{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run PII detection on the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Use Regexes for all the PII types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration bigcode--pii-for-code-2f55abc831915fa6\n",
      "Found cached dataset json (/Users/loubnabenallal/.cache/huggingface/datasets/bigcode___json/bigcode--pii-for-code-2f55abc831915fa6/0.0.0/e6070c77f18f01a5ad4551a8b7edfba20b8438b7cad4d94e6ad9378022ce4aab)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1f35e1c560c4fa2b80506945bd46695",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "from pii_detection import scan_pii_batch\n",
    "\n",
    "from utils.evaluation import evaluate_pii, evaluate_pii_ds, recall_precision\n",
    "#from utils.jia_evaluation import evaluate_pii, evaluate_pii_ds, recall_precision\n",
    "\n",
    "ds = load_dataset(\"bigcode/pii-for-code\", use_auth_token=True)[\"train\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['content', 'language', 'license', 'path', 'annotation_id', 'pii', 'pii_modified', 'id'],\n",
       "    num_rows: 400\n",
       "})"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0324b48adfc94b0db3997a7e4d7bbea3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/40 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "examples = ds.select(range(400))\n",
    "ds_pii_2 = examples.map(lambda x: scan_pii_batch(x, key_detector=\"other\"), batched=True, batch_size=10, load_from_cache_file=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate on the whole 100 samples\n",
    "metrics_2, metrics_dict_2, details_2 = evaluate_pii_ds(ds_pii_2, pred_column='secrets', ref_column=\"pii_modified\", overall_score=False, alpha=0.8, beta=0.8, return_details=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'EMAIL': {'recall': 0.9585492227979274, 'precision': 0.8767772511848341},\n",
       " 'IP_ADDRESS': {'recall': 0.8282828282828283, 'precision': 0.803921568627451},\n",
       " 'KEY': {'recall': 0.5294117647058824, 'precision': 0.782608695652174}}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'EMAIL': {'TP': 185, 'FN': 8, 'FP': 26},\n",
       " 'IP_ADDRESS': {'TP': 82, 'FN': 17, 'FP': 20},\n",
       " 'KEY': {'TP': 18, 'FN': 16, 'FP': 5}}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_dict_2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following data is used to build a vizualization space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's just get emails\n",
    "email_detections = {\"FN\": [], \"FP\": []}\n",
    "TAGS = [\"EMAIL\", \"KEY\", \"IP_ADDRESS\"]\n",
    "\n",
    "def get_fn(data, tag):\n",
    "    detections = []\n",
    "    for i in range(len(data)):\n",
    "        details = data[i][\"details\"][tag]\n",
    "        if not details[\"FN\"]:\n",
    "            continue\n",
    "        # sort elements inside FN by start index\n",
    "        details[\"FN\"] = sorted(details[\"FN\"], key=lambda x: x[\"start\"])\n",
    "        print(f\"not skipped {i}\")\n",
    "        # add content to each detection while hightlighting detected words\n",
    "        subparts = []\n",
    "        advance = 0\n",
    "        for detection in details[\"FN\"]:\n",
    "            if detection[\"value\"] == \"discoiquuid\":\n",
    "                print(\"wrong label found\")\n",
    "                continue\n",
    "            text = details[\"content\"][advance:detection[\"start\"]] \n",
    "            subpart = text if text else \" \"\n",
    "            subparts.append(subpart)\n",
    "            subparts.append(\"PI:FN:\" + details[\"content\"][detection[\"start\"]:detection[\"end\"]] + \"END_PI\")\n",
    "            print(tag, \"PI:FN:\" + details[\"content\"][detection[\"start\"]:detection[\"end\"]] + \"END_PI\")\n",
    "            last_part = details[\"content\"][detection[\"end\"]:]\n",
    "            advance = detection[\"end\"]\n",
    "        displayed_text = \"\".join(subparts) + last_part\n",
    "        detections.append(displayed_text)\n",
    "    return detections\n",
    "\n",
    "def get_fp(data, tag):\n",
    "    detections = []\n",
    "    for i in range(len(data)):\n",
    "        details = data[i][\"details\"][tag]\n",
    "        if not details[\"FP\"]:\n",
    "            continue\n",
    "        # sort elements inside FP by start index\n",
    "        details[\"FP\"] = sorted(details[\"FP\"], key=lambda x: x[\"start\"])\n",
    "        print(f\"not skipped {i}\")\n",
    "        # add content to each detection while hightlighting detected words\n",
    "        subparts = []\n",
    "        advance = 0\n",
    "        for detection in details[\"FP\"]:\n",
    "            if detection[\"value\"] == \"discoiquuid\":\n",
    "                print(\"wrong label found\")\n",
    "                continue\n",
    "            text = details[\"content\"][advance:detection[\"start\"]] \n",
    "            subpart = text if text else \" \"\n",
    "            print(f\"length of subpart: {len(subpart)}\")\n",
    "            subparts.append(subpart)\n",
    "            subparts.append(\"PI:FP:\" + details[\"content\"][detection[\"start\"]:detection[\"end\"]] + \"END_PI\")\n",
    "            print(tag, \"PI:FP:\" + details[\"content\"][detection[\"start\"]:detection[\"end\"]] + \"END_PI\")\n",
    "            last_part = details[\"content\"][detection[\"end\"]:]\n",
    "            print(f\"length of last part: {len(last_part)}\")\n",
    "            advance = detection[\"end\"]\n",
    "        displayed_text = \"\".join(subparts) + last_part\n",
    "        detections.append(displayed_text)\n",
    "    return detections\n",
    "\n",
    "print(\"email FN\")\n",
    "email_detections_fn = get_fn(details_2, \"EMAIL\")\n",
    "print(\"email FP\")\n",
    "email_detections_fp = get_fp(details_2, \"EMAIL\")\n",
    "\n",
    "print(\"key FN\")\n",
    "key_detections_fn = get_fn(details_2, \"KEY\")\n",
    "print(\"key FP\")\n",
    "key_detections_fp = get_fp(details_2, \"KEY\")\n",
    "\n",
    "print(\"ip FN\")\n",
    "ip_detections_fn = get_fn(details_2, \"IP_ADDRESS\")\n",
    "print(\"ip FP\")\n",
    "ip_detections_fp = get_fp(details_2, \"IP_ADDRESS\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save list in json file\n",
    "import json\n",
    "with open(\"data/email_detections_fn.json\", \"w\") as f:\n",
    "    json.dump(email_detections_fn, f)\n",
    "\n",
    "with open(\"data/email_detections_fp.json\", \"w\") as f:\n",
    "    json.dump(email_detections_fp, f)\n",
    "\n",
    "with open(\"data/key_detections_fn.json\", \"w\") as f:\n",
    "    json.dump(key_detections_fn, f)\n",
    "\n",
    "with open(\"data/key_detections_fp.json\", \"w\") as f:\n",
    "    json.dump(key_detections_fp, f)\n",
    "\n",
    "with open(\"data/ip_address_detections_fn.json\", \"w\") as f:\n",
    "    json.dump(ip_detections_fn, f)\n",
    "\n",
    "with open(\"data/ip_address_detections_fp.json\", \"w\") as f:\n",
    "    json.dump(ip_detections_fp, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAHWCAYAAAD6oMSKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHiklEQVR4nO3dd3wVVeL///dNbnpIAiQhCWAooXdUQCAgxYJEIUgRRJHOggrLsq4CFpAirhRl8bOugsDKopQPRRexEZCiP6QpCAoiiEAioYQIAdLO7w+/mQ+XBEjITWF4PR+PPB6ZM2dmzkzuufedKec6jDFGAAAAuOl5lHQDAAAA4B4EOwAAAJsg2AEAANgEwQ4AAMAmCHYAAAA2QbADAACwCYIdAACATRDsAAAAbIJgB1s7dOhQSTcBAIBiQ7CD7Rw6dEijRo1SzZo11aRJk5Juzg1ZunSpRowYoVatWqlChQoaN25cSTcJt7gffvhBs2fP1osvvlgi2//22281bNgw9erVS2XKlFGPHj109uzZEmlLUTLGaOfOnXr55Ze1atWqkm4ObkYGuI5FixaZu+++20gyksysWbPyrLdu3TozaNAg43Q6jSRzzz33mEWLFhVza43JysoySUlJxt/f3wQHBxf79gvro48+MpGRkSYzM9NcunTJtGvXzjidTnP27NlcdYODg62/S3BwsClfvrx1/P38/Ez58uVNQECAVadLly5m8+bNZuDAgcbT09NIMk6n09SrV8/ExMSYChUqmNjYWDN16lRz+vTpG2r/5MmTTYsWLaxtlitXztSrV8/6qVWrlilfvryRZKKjo0tdez766CPTtWtXq/6oUaPyXO/27dvNyJEjrePbrFkzM3v27BtqY2n3zjvvmI4dOxpJpm3btsW+/V9//dWULVvWfP/998YYY8aNG2ckmTVr1hR7W4rae++9Z+rXr28kmZkzZ5Z0c3ATItghX7KyskybNm2MJOPp6Wn++9//XrXuyJEjTe3atU1mZmYxtjC36OjomzLYNW3a1LRu3dqavnjxojl8+HCedYODg83LL79sUlNTrbK2bdsaSebFF1+0yo4ePWq6dOliunTpYpXdc889RpLp3bu3VXb27Fkzffp043Q6TXh4uPn6669vaB+OHTtmBaNPPvkkzzqfffaZiYmJKbXtefTRR61l/vnPf1513TNnzjTlypVz+RvY0d69e0ss2I0ePdpIcnlP2b9/f7G3o7i8/fbbbgl269evN7/99pt7GoWbBpdikS8eHh6qWrWqgoKClJWVpUceeUR79uzJs27dunXVuHFjeXp6FnMrb35paWnasWOHy7Hz8fFRdHR0nvXr1aun8ePHq0yZMtdcb8WKFfXPf/5TTqfTKouMjJQkeXt7W2VBQUEaPXq0nn76aZ04cUL9+vW7of3IWfeV679cx44d1blz51LbnpiYGAUFBUmSnnzySX3xxRd5Lle3bl3VrFnzun+Dm52fn1+JbXvTpk2S5NIvatSoUVLNKXKX99MblZSUpN69eystLc0NLcLNhGCHAnn55ZdVv359/f7774qLi9OJEydy1fH29paPj08JtO7md/r06QLVHzlyZL7rRkREqE+fPta0w+G4at327dtLkn788cc8/8bXc611X27WrFmltj3SH4GuQ4cOyszMVPfu3fXjjz/mWobXe9E7depUSTfhpnLq1Cl17txZiYmJJd0UlACCHQokKChIa9asUWRkpH755RfFx8fr0qVL11xmy5Ytat++vRwOh5544glJUnZ2thYsWKA77rhDDodDL730kiQpKytLGzZs0MiRI1WlShWlpaVp+vTpqlGjhnx9fdWqVSv98ssvysrK0qxZs1SnTh0FBgbqwQcf1JkzZ/Lc/vHjx9WzZ0+FhIQoJCREvXr10tGjR3PV++CDD9S5c2c1a9ZMFSpUUI8ePXT48GFr/m+//aY5c+aoWbNmSklJ0bhx4xQUFKQHHnjgusdt7dq16tGjh/r376+4uDg1bNhQ06dPV2ZmplWnR48e1hmp3bt3Ky4uTnFxcVq2bNlV19uzZ8/rbvty3bp1y1e9nJvSHQ6HfH19C7SN/Ni5c2euEFUa2+Pl5aXly5erXr16SklJUVxcXIHD9+V+++03DRkyRC+//LL+9re/qXLlyqpSpYpLnUOHDmnIkCEaOXKkxowZo9jYWP39739XVlaWVeebb77R2LFjVbt2be3fv1/z589XgwYN5Ovrq0aNGmnnzp2SpIULF6pp06YKCAhQbGysjhw5Yq3j+PHjmjVrllq0aKEvv/xSc+fOVZ06deTr66smTZros88+y/d+ff755+ratatatmyp8PBw3X///fr2229d6nzzzTfq16+fXnnlFQ0bNkz+/v7W+8HVTJw4UXFxcTp+/LgkWX3i8gc4Lly4oEmTJunhhx/WgAEDdMcdd6hTp07WWb4c+/bt0wsvvKCuXbsqNTVVXbt2VUBAgKZMmXLNNqSmpuqvf/2r7rvvPsXExKh69eqaNGmSS9+VpFWrVqlnz54aP368BgwYoLvvvltff/11rvWdO3dO48aNU8+ePdW3b181bdpUzz33nH7//fc8t79lyxbFxsYqICBANWrU0KJFi67ZXkl68cUXrfeugQMHKi4uzjrjnJmZqZkzZ6pPnz4aO3asunXrpl69eunYsWO51pOYmKjHHntM/fr1U+fOndW9e3dt3br1uttHCSvpa8G4efTr18+8++67xpg/bhzPuWm8T58+LvXeffdd069fP5eyjz/+2EjKVf7WW2+53A+WnJxsli9fboKCgowkM3DgQPPmm2+ab7/91syYMcNIMu3atTODBg0yS5cuNd9++63585//bCSZkSNHuqw7OjraBAYGmrZt25oXX3zRvPbaa+b222+3bpK//Gb8SZMmmb59+5pLly4ZY4zZunWr8fT0NBUrVjQpKSnm6NGjZvbs2dY+jxs3zkyePNnUrl3bVKlS5ZrH7ZVXXjFhYWHm4MGDVtmqVauMh4eH6dSpk8nIyLDKDx06VOj7mPK6xy4v/fr1y/NvYowxDz30kJFkOnbseMPt0P+7Py0hIcGlPCUlxfTp0yfX/UOlrT0vvviidQx/+eUXExERYf1t0tPTrXoJCQn5/nt16tTJTJkyxZpOTEw0TZs2dWlLeHi4efDBB62yLVu2GElm2rRpxhhj0tLSzMqVK010dLR1T+K0adPMjh07zKJFi4ynp6epXbu2GTVqlJk7d6757rvvzLRp06yHZ4wx5sKFC+bvf/+7CQ8PN5LMfffdZ0aPHm0WL15sRo4caTw8PIyXl5fZtGmT1Y6rvTYXLlxo7rnnHusew0OHDpmgoCATFBRkvebPnz9vQkNDzZYtW6zlPvnkE/PEE0/k67jl7OuVzp07Z+68807Ts2dPqx+lp6eb3r17G09PT7Nw4UJjzB/9+bnnnjOSTP369c2f/vQnM2nSJBMcHHzNNpw7d87cfvvt5sMPP7TKRo4caSSZESNGWGX/+c9/jCSzfPlyq2zIkCHG19fX5f7YEydOmHr16rncs7lgwQIjyTzwwANW2bvvvmskmc6dO5vevXub9evXm1WrVpmwsDDjdDrNoUOHrnvMct4Hrqw7ZMgQI8mcOnXKGGNMZmamqVOnjqlbt65LvZSUFFOtWjUzduxYq+yRRx4xkkxERISpVavWVe9XRcki2CHfLg92xvzx9GbOk4wTJ060yvMKdgkJCXl+aOe8gV0ZQlq1amUkme3bt7uU57zBb9261Sq7ePGi8fHxcbnx/fK6O3bssMouXLhgmjRpYiSZ0aNHG2OM+f77742Xl5c5efKky/IdOnQwkswbb7yRq105Hxjp6ekmJSUlj6P1h507dxqHw2FeeumlXPMGDRpkJJnp06dbZSUZ7LKysswPP/xgHn/8cSPJ1KhR46oPbeRHTpCqUKGCiY6ONtHR0VY4Uh43hpe29lwe7IwxZtu2bVawHzBggFVekGAXGBhonnrqKZeyN9980/p9165dRpKJj4+3yjIzM40k06lTJ5flch7uuDxMGPN/f/8lS5a4lFepUsV4eXmZ7OzsXOuYP39+rn2XZNq0aWOV5fXaPHnypAkICDC7du1yWX7gwIEufWzbtm15tvXyfb+WqwW7nH/qrnxdpKammnLlyhlfX19z7NgxY4wxGRkZ1lPXP//8s1Xv4sWLV93u3/72N9O1a1eXsl9//dVIMl5eXlY4GjVqlJFkVq1aZdV75513jCTzwQcfWGXdunUzsbGxLuv7/fffTeXKlU2zZs2sspz3xcsfJDLGmPHjxxtJ5p133rlqm3NcLdg1btzYeHp6uvxj27dvXyPJ5UGL6dOnG0lm7dq1Vtn27dvzbBdKFy7F4oZ17txZb7zxhqQ/Tv0vXbrUbevOuXm4XLlyLuWVK1eWJIWFhVllPj4+CgsLsy7XXOnysex8fX31zDPPSJJWrlwpSfr3v/8tp9OpgQMHqmvXrtbP+fPnVatWLZexsnLaFRsbK+mPS3XBwcFX3Y8FCxbIGKPGjRvnmvf4449bdUrSihUrVKtWLUVHR+v+++/X4cOHNXnyZO3YseOqD20UxPvvv6/Dhw/r8OHDSkxMVGpqqvr373/TtCfH7bffrsWLF8vDw0Pz5s3T3//+9wJv+/bbb9fs2bM1atQo69Lbn/70J2t+o0aNtGnTJr399tuS/hjTbOPGjZL+uOR4uYL0EUmqVKmSMjIydPLkyVzruPK4Pv300/L29tbmzZtzbfdyy5Yt0/nz5/Xiiy+69J39+/erVq1a1o37MTExCg4O1mOPPaZ//etfys7OzrXvBZVzO0dISEiu9pcpU0Zdu3bVxYsX9cEHH7jsa8WKFVW1alWr3rXuj1ywYIH27dvnsm9PPvmk6tSpo2rVqln3sE2cOFEJCQl68MEHJf1x+XbXrl2S/u/vduzYMa1YsUL33nuvyzYCAwN18OBBbd68Odf2a9as6TKdc9m+MPfOrVy5Utu3b1fZsmUlSUeOHLEu217+t865nzQ1NdUqa9q0qXx9fa1L/SidCv/oDW5pw4cP18GDBzVjxgz169fPLR+813K1J209PT3z/fRX8+bNJcm6z27Pnj0KCQmxgp47/fDDD5KU531hDRo0kCTt37/f7dstiPj4eM2fP7/YtlemTBk9/fTTWr9+/U3Rnss9+OCDmjVrlp5++mk9++yzqlmz5jWD/ZXmzZunjh076vXXX9cHH3ygadOmWQE/R6tWrZSSkqJXX31VJ0+e1P333y/pj5CXH9fqI5J0/vz5XKHvSuXKlVN0dLQOHDig06dPq2LFinnWy3kyfsmSJVd92liSgoODtWjRIvXo0UNDhw7VW2+9pTlz5qhFixb52aU8JScn6/Tp04qIiMhzfmH716lTp5SUlKT+/ftf9z68MmXK6O6779auXbv073//W2FhYdYxzvm77dixQ8aYXEFc+uMfxPzI+RteeX9fQURHR+u2227Thx9+qM8//1z16tXL1VZJql69uiTpq6++Uo8ePazyMmXKKCoq6oa3j6LHGTsU2t///nd169ZNFy5cUJcuXfTrr7+WdJOuKWfoi5zhKdLT03Xy5EllZGS4fVs5b8S//fZbrnn+/v6S8v+mbieNGjUq1NkadytIe5566imNGjVK2dnZevTRR60zM/lRrVo17dy5U08++aROnjypfv36KS4uzuWfksWLF6tBgwa666679Oqrr1pPBBe3cuXKyeFwKCQk5Kp10tPTJemqZ8sv17lzZ+3atUv33XefduzYoVatWhXqWyxy+tbp06fzDDqF7V8F2bcLFy6of//+GjVqlMaPH69nn31Wt912m0udnDO0Jf01h4cPH9Zdd92ldevWacaMGRoyZIg1rM/lhgwZooYNG2ru3Ln67rvvJElffvmlzp07V2LfPoL8Idih0Dw8PPTee++pRYsWSkpK0ssvv1zSTbqm5ORkSbLOFuRcolq9enWe9d977z2XJxILomHDhpKk/+//+/9yzcsJe3ldprU7h8NRqoYIKWh7pk+frvj4eJ0/f966tJ8f3333nYKDgzV79mzt27dP7du313//+1/94x//kPTHB2ffvn31+OOPW5f7S0piYqIaNGiggICAq9apVKmSJGn58uV5zl+1apVSUlKUmJiokydPqmbNmlq7dq2++OILRUdHa+LEiTf8lGVoaKiioqKUnp6e56XBwvav0NBQ+fj4aO3atXleDbh06ZLef/99SX+E/fnz5+vtt9+2LnFeKecS+dKlS/McSWDfvn2FeuI6PzIzM3X//ffr2LFjmj59+jXHGg0JCdGWLVtUp04dvfTSS/rzn/+sNWvWaO/evWrTpk2RthOFQ7BDvl28ePGqQ5v4+flp9erVqlatWp5nvnLOjl15b0jOfRw3GpxuxJdffilJGj16tCRZg9KOHDky13/TS5cu1bfffnvDgy0PGDBAXl5eWrp0qc6fP+8yL+feqWHDht3Quq8m53LK9S7d5bdejsOHD191UOorXf73zO/6S1t7rvV69/Dw0KJFi9SsWbMCnemdNGmS9XtMTIw++ugjBQYGWmeFlixZouzsbJdLpUlJSbn2oajt2bNHR44cue44iTlD/UyYMEHbt293mbdx40YtX75cISEhOnjwoP7zn/9Y89q3b2/dW5qfM2JXM3ToUEnSu+++m2vexo0bVb58eXXv3v2G1u3l5aV7771Xv/32m/70pz+5HP/09HQ9/vjjql27tiRZAe9af7cWLVooKipKv/zyi5555hnrPkPpj9fyzJkz87xMe6Nybv+4/H1n165d+vHHH1W+fHl5ePzfx39er7GLFy+qX79++ve//63//d//1cyZM/XKK6/kGp4HpQ/32CHfvv/+e5UvX/6q88PCwrRmzRq1bNky17y6desqPDxcn332mSZMmKCGDRtq7dq11pvbxx9/rDZt2uiee+6RJOs/5Cs/zHI+aK8c8ymnfkZGhnXpJedSzJ49e1S/fn1J0s8//6wXXnhBU6dOtS5xde3aVe3bt9e6devUqFEjde/eXeHh4dqxY4fOnj3r8o0DOTcXX35D8bXExMRozpw5Gjp0qMaMGaM333xTDodD58+f17Rp09S/f3+XQYNzBmK9/Ab3gsoZwDevy7+Xyzk7cL160h/HrVatWsrMzNSUKVP03HPPXbP+5R/W+b3Ru7S15/vvv3f58L2Sn5+fPvzwwwLdJ7Z69Wpt2bLF6iNpaWnKzs62xhfMuXfp9ddf12233SZjjD755BP5+/tr9+7dWr16tSpWrKjbb7+9UH3kSvPmzdNdd90lHx8fXbhwQaNHj1b37t1dHirJGSfy8hvsb7/9dj3xxBOaP3++WrZsqYcffljR0dH64Ycf9P3332vDhg1W3VdffVWPPvqo9R5y9uxZVahQ4bpnJo0x1rZPnjyp0NBQa96zzz6rDRs26O2331bv3r2tdX355ZfatGmTVqxYYV1mzGn31caLy8vUqVOVkJCghQsX6ptvvtEDDzyg9PR0ffTRRxo6dKh1NjAqKkoHDhzQ8OHD1b9/f+3Zs8e6Z/Ojjz6Sp6enHn/8cb355pt6+OGH9cYbb+jTTz9V27ZtlZKSop07d+rzzz+3tpvz/pKSkuLSnpzjcO7cueu2vWbNmvrkk0/0zjvv6NFHH9XmzZv18MMPy8PDQ99++62mTJmiO++8U59//rl1FeO9995TgwYN1K1bN82fP1/Lly9XUlKSKlSoYA3y7ePjo6ZNm2rEiBFFMqYk3KD4H8TFzWbevHnW98Q6HA7To0cPl+FGrrRhwwYzZMiQXOUbN240TZo0MX5+fqZx48Zm5cqVZuvWraZDhw7mrbfeMkeOHDFHjx41U6dOtYZReeKJJ8ymTZtMYmKimT17tvH19TWSzIMPPmjWr19vjh8/bv76179aQ1aMHDnSGnrhhx9+MIMHDza1a9c27dq1M506dTJdunQxn376aa62nT9/3owZM8ZUqlTJ+Pj4mBo1aphx48aZs2fPGmP+GMfsL3/5i9Wu9u3bm7lz57oMHXEtn3/+uWnfvr1p3bq1GTJkiOnevbuZO3euS5358+dbQ6xIMmPGjDErV67M1/qN+WOIhP79+1vLBwYGmr/+9a+5vih9y5YtZsyYMcbHx8fo/33377Bhw3INj3G548ePm0qVKhlJpl69etdsx6uvvmoNCyPJVK1a1bzwwgtXHXurtLVn5cqVJi4uzqp///33X3O8rn379rkMT3Itnp6extfX1/Tv3988//zzpkePHi5DZKSmppr4+HgTEBBgIiMjzZ///Gdz9uxZM2bMGOPr62seeughc+bMGfPGG29YYz3GxcWZtWvXmtTUVPOvf/3LGpuuZcuWZvXq1SY1NdVMnjzZOJ1OI8k8+uij1vh0OUPMjB071rRv39507tzZtGnTxkybNs1lfMUPPvjA3HXXXdbfZ9KkSeaHH34wxvwxHMvkyZNN9erVjbe3t6lataoZMWKESUpKspbfuHGjkWSioqLM008/bcaOHWt69uxp9uzZc83j9emnn5phw4ZZf4suXbqYd955xxpv0hhjLl26ZCZPnmwaNGhgunfvbgYPHmweffRR8+2331p1Nm/ebLp162atZ+jQobn6xdXs2rXLPPDAA6ZMmTImODjYtGzZ0qxYscKlzpdffmnq1q1rfH19TZMmTczixYvNmTNnTJ06dUy5cuXMrFmzrLpffPGFad26tfHz8zNRUVFm8ODBJjEx0Zq/dOlSU6tWLSPJhIWFmZkzZ5rs7GyzYMECU61aNWscucuHnsrLoUOHTKNGjUxAQIDp2bOnNZzTnDlzTGRkpAkICDD33Xef2bZtm9myZYspV66cqV69ulm3bp0xxpgzZ86YTp06mebNm5uoqCjj7+9vPDw8rGP40EMP5ev4ofg5jMnndQkAt7yTJ09q4MCBWrVqVUk3RVLpa8/N5oknntCCBQuUkJCgu+++u6Sbg1Lkww8/1Pr16zV9+nSX8rS0NB08eFCPP/64NmzYkOeDFyhZXIoFkG+vvPKKxo8fX9LNsJS29gB2sH//fvXs2VM//fRTrnn+/v5q0KCB7rvvvhu+9xhFi4cnAOTLjBkzdM899+jOO+8s6aZIKn3tAexi9erVunjxost3C1/ul19+kZ+f3zWfmEbJ4YwdgHzJeYq4tCht7bkZ5dyEn/PQDiBJbdq0kdPpVOfOnTV8+HA1a9ZMAQEBSkpK0vr16+V0OvX666+XdDNxFdxjBwC3mNOnT2vmzJmaPn26Lly4oHr16ql///7q16+fy1OnuHVt3bpVr776qjZv3qxTp04pLCxMLVu21NChQ9WxY8eSbh6ugWAHAABgE9xjBwAAYBMEOwAAAJsg2AEAANgEwQ4AAMAmburhTs6cOaPMzMySbkapFhYWZn0PIIDCo08B7kWfuj6n06myZcvmr24Rt6VIZWZm5vmF1vhDzpc2Z2ZmioefgcKjTwHuRZ9yPy7FAgAA2ATBDgAAwCYIdgAAADZBsAMAALCJm/rhCQAAcPMyxujo0aM6d+5cSTelxPn4+MjHx6fQ6yHYAQCAEvH7778rICBAQUFBJd2UEmWM0YULF3T+/HkFBAQUal1cigUAACXCGOOWs1Q3O4fDIX9/f7eMzUuwAwAAKAVyxvUrDIIdAACATRDsAAAAbIKHJwAAQKkR+5/YYt3exj4b8123YsWK15w/evRo9ezZUy1atLDKQkJC1LBhQ40bN07169e/4XbmF8EOAAAgH3bu3Gn9vnr1ar322mv68ssvrbKAgACdPn1akvT++++rVq1aSkxM1PPPP6++fftqw4YNCg4OLtI2cikWAAAgH8LDw62fMmXKyOFwuJRdPlRJ2bJlFR4erkaNGun5559XcnKySzAsKgQ7AACAIuTn5ydJSk9PL/JtEewAAACKyNmzZzVr1iwFBASoSZMmRb497rEDAABwsy5dusjDw0NpaWmKjo7W//zP/ygsLKzIt0uwu47ifjrnSgV5WgcAAJQO//M//6OaNWuqbNmyRf7AxOUIdgAAAG4WFRWlKlWqFPt2uccOAADAJgh2AAAANsGlWAAAUGrcLPeW9+rVS7169cpVXrlyZR07dqwEWvQHztgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJvhKMQAAUGr0nZRZrNt7b3zBotCoUaO0dOnSXOWbNm3S66+/bs3z8vJSxYoV1b17dz311FNyOosnchV4K3v37tXq1at16NAhnTlzRmPGjFGzZs2s+T179sxzub59++qhhx6SJI0YMULJycku8/v06aOuXbsWtDkAAADFql27dpoxY4ZLWfny5V3mpaen64svvtC4cePkdDr11FNPFUvbChzsLl26pCpVqqh9+/Z67bXXcs3/17/+5TK9c+dO/fOf/1Tz5s1dynv27KmOHTta076+vgVtCgAAQLHz9vZWeHj4def169dPa9eu1aefflp6g12TJk3UpEmTq84PCQlxmf7mm29Ur149VahQwaXcz88vV10AAAA78fX11ZkzZ4pte0X68ERKSop27typ9u3b55q3cuVKDRgwQM8884xWr16trKysomwKAACAW3z++eeqUaOG9TNkyJBcdYwx+vLLL7Vhwwa1atWq2NpWpHfybdiwQb6+vi734ElSp06dVLVqVQUGBurHH3/U4sWLdebMGfXr1y/P9WRkZCgjI8Oadjgc8vPzs363s8LsX86ydj9GQHGhTwGQpJYtW2rq1KnWtL+/v/V7TujLzMxUdna2unbtqr/85S/5Xndh31+KNNglJCQoNjZW3t7eLuVxcXHW79HR0XI6nXr77bfVp08feXl55VrPihUrtGzZMmu6atWqmjZtmsLCwoqu8aVEZGRkodcRERHhhpYAyEGfAtzjwoULknTFZ3/xPhWbV+64Fg8PDwUEBKhmzZp5zmvVqpVeffVVeXt7KyIiokBPw3p7exf6c7/Igt2+fft0/PhxjRo16rp1a9SooaysLCUnJysqKirX/Pj4eJcwmJNmk5OTlZlZvC+A4paYmHjDyzocDkVERCgpKUnGGDe2Crg10acA90pPT5ckl6tyxa2g287OzpYxJs/lsrOz5efnp8qVK0vSVetdTXp6ep6f+06nM98ns4os2K1bt07VqlVTlSpVrlv38OHDcjgcCgoKynO+l5fXVRO13d9c3bF/xhjbHyegONGnABSVwr63FDjYXbx4UUlJSdb0iRMndPjwYQUGBio0NFSSlJaWpq+//lqPPfZYruX379+vAwcOqF69evLz89P+/fu1YMECxcbGKjAwsBC7AgAAcGsrcLA7ePCgJkyYYE0vXLhQktS2bVuNGDFCkrRlyxYZY9S6devcG3Q6tWXLFi1dulQZGRkKDw9X586dXS61AgCAW1NBvwmiuM2aNeuG5hUXh7mJryckJycX+XX52P/EFun6r2djn403vKzD4VBkZKQSExO5bAS4AX0KcK/U1FSVL1++RO+xK01SU1PzvC3Ny8sr3/fYFek4dgAAACg+BDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATpft7OwAAwC0lanHFYt3e8d7HClR/1KhRWrp0aa7yTZs26fXXX9fSpUv13HPP6cknn7TmrV27VgMHDtSxYwXb1o0g2AEAABRAu3btNGPGDJey8uXLS5J8fX315ptvqm/fvgoJCSn2tnEpFgAAoAC8vb0VHh7u8uPp6SlJat26tcLCwvSPf/yjRNpGsAMAAHATT09PPfvss3r33Xd1/PjxYt8+wQ4AAKAAPv/8c9WoUcP6GTJkiMv8Tp06qW7dupo+fXqxt4177AAAAAqgZcuWmjp1qjXt7++fq864cePUs2dPDRs2rDibRrADAAAoCH9/f1WtWvWadVq0aKG2bdtq6tSp6tmzZzG1jGAHAABQJMaOHat7771X1atXL7Ztco8dAABAEahTp47i4+M1b968YtsmwQ4AAKCI/PWvf1V2dnaxbY9LsQAAoNQo6DdBFLdZs2YVaF7lypV16NChomvQFThjBwAAYBMEOwAAAJsg2AEAANgEwQ4AAMAmCHYAAAA2QbADAACwCYIdAAAoMcaYkm5CqeCuse4IdgAAoET4+PgoLS2tpJtR4rKzs/X777/L39+/0OtigGIAAFAifH195XA4dObMGTkcjpJuTokKCAiQ01n4WEawAwAAJSYiIkLGGC7JugmXYgEAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbKPB3xe7du1erV6/WoUOHdObMGY0ZM0bNmjWz5s+ZM0cbNmxwWaZRo0YaN26cNX3u3DnNmzdP27dvl8PhUPPmzdW/f3/5+voWYlcAAABubQUOdpcuXVKVKlXUvn17vfbaa3nWady4sYYPH/5/G3G6buaNN97QmTNnNH78eGVlZenNN9/UW2+9pZEjRxa0OQAAAPh/ChzsmjRpoiZNmlx7pU6nQkJC8px39OhR7dq1S1OnTlX16tUlSQMGDNDUqVP12GOPqVy5cgVtEgAAAHQDwS4/9u7dq0GDBikgIED169fXI488ojJlykiS9u/fr4CAACvUSVKDBg3kcDj0008/uVzWzZGRkaGMjAxr2uFwyM/Pz/rdzgqzfznL2v0YAcWFPgW4F33K/dwe7Bo3bqzmzZsrPDxcSUlJWrx4saZMmaLJkyfLw8NDKSkpCgoKclnG09NTgYGBSklJyXOdK1as0LJly6zpqlWratq0aQoLC3N380udyMjIQq8jIiLCDS0BkIM+BbgXfcp93B7sWrVqZf1+2223KTo6Wk899ZS+//57NWjQ4IbWGR8fr7i4OGs6J9knJycrMzOzcA0u5RITE294WYfDoYiICCUlJckY48ZWAbcm+hTgXvSp/HE6nfk+mVUkl2IvV6FCBZUpU0ZJSUlq0KCBQkJClJqa6lInKytL586du+p9eV5eXvLy8spznt1fCO7YP2OM7Y8TUJzoU4B70afcp8jHsTt16pTOnTunsmXLSpJq1qyp8+fP6+eff7bq7NmzR8YYxcTEFHVzAAAAbKvAZ+wuXryopKQka/rEiRM6fPiwAgMDFRgYqKVLl6p58+YKCQnRb7/9pvfee08RERFq1KiRJKlSpUpq3Lix3nrrLQ0ePFiZmZmaN2+eWrZsyROxAAAAhVDgYHfw4EFNmDDBml64cKEkqW3btho8eLCOHDmiDRs26Pz58ypXrpwaNmyoXr16uVxKffrppzV37lxNnDjRGqB4wIABbtgdAACAW5fD3MQXtZOTk12GQSkKsf+JLdL1X8/GPhtveFmHw6HIyEglJiZy7wLgBvQpwL3oU/nj5eWV74cn+K5YAAAAmyDYAQAA2ATBDgAAwCYIdgAAADZBsAMAALAJgh0AAIBNEOwAAABsgmAHAABgEwQ7AAAAmyDYAQAA2ATBDgAAwCYIdgAAADZBsAMAALAJgh0AAIBNEOwAAABsgmAHAABgEwQ7AAAAmyDYAQAA2ATBDgAAwCYIdgAAADZBsAMAALAJgh0AAIBNEOwAAABsgmAHAABgEwQ7AAAAmyDYAQAA2ATBDgAAwCYIdgAAADZBsAMAALAJgh0AAIBNEOwAAABsgmAHAABgEwQ7AAAAmyDYAQAA2ATBDgAAwCYIdgAAADZBsAMAALAJgh0AAIBNEOwAAABsgmAHAABgEwQ7AAAAm3AWdIG9e/dq9erVOnTokM6cOaMxY8aoWbNmkqTMzEy9//772rlzp06cOCF/f381aNBAffr0Ubly5ax1jBgxQsnJyS7r7dOnj7p27Vq4vQEAALiFFTjYXbp0SVWqVFH79u312muvucxLT0/XoUOH9PDDD6tKlSo6d+6c5s+fr1dffVWvvPKKS92ePXuqY8eO1rSvr+8N7gIAAACkGwh2TZo0UZMmTfKc5+/vr+eff96lbMCAARo7dqxOnjyp0NBQq9zPz08hISEF3TwAAACuosDBrqDS0tLkcDjk7+/vUr5y5UotX75coaGhat26tTp37ixPT88815GRkaGMjAxr2uFwyM/Pz/rdzgqzfznL2v0YAcWFPgW4F33K/Yo02KWnp2vRokVq1aqVS7Dr1KmTqlatqsDAQP34449avHixzpw5o379+uW5nhUrVmjZsmXWdNWqVTVt2jSFhYUVZfNLhcjIyEKvIyIiwg0tAZCDPgW4F33KfYos2GVmZmrmzJmSpEGDBrnMi4uLs36Pjo6W0+nU22+/rT59+sjLyyvXuuLj412WyUn2ycnJyszMLIrmlxqJiYk3vKzD4VBERISSkpJkjHFjq4BbE30KcC/6VP44nc58n8wqkmCXE+pOnjypF154Iddl2CvVqFFDWVlZSk5OVlRUVK75Xl5eeQY+SbZ/Ibhj/4wxtj9OQHGiTwHuRZ9yH7ePY5cT6pKSkvT888+rTJky113m8OHDcjgcCgoKcndzAAAAbhkFPmN38eJFJSUlWdMnTpzQ4cOHFRgYqJCQEM2YMUOHDh3S3/72N2VnZyslJUWSFBgYKKfTqf379+vAgQOqV6+e/Pz8tH//fi1YsECxsbEKDAx0244BAADcagoc7A4ePKgJEyZY0wsXLpQktW3bVj169NC2bdskSc8884zLci+++KLq1asnp9OpLVu2aOnSpcrIyFB4eLg6d+7scg8dAAAACs5hbuKL2snJyS7DoBSF2P/EFun6r2djn403vKzD4VBkZKQSExO5dwFwA/oU4F70qfzx8vLK98MTfFcsAACATRDsAAAAbIJgBwAAYBMEOwAAAJsg2AEAANgEwQ4AAMAmCHYAAAA2QbADAACwCYIdAACATRDsAAAAbIJgBwAAYBMEOwAAAJsg2AEAANgEwQ4AAMAmCHYAAAA2QbADAACwCYIdAACATRDsAAAAbIJgBwAAYBMEOwAAAJsg2AEAANgEwQ4AAMAmCHYAAAA2QbADAACwCYIdAACATRDsAAAAbIJgBwAAYBMEOwAAAJsg2AEAANgEwQ4AAMAmCHYAAAA2QbADAACwCYIdAACATRDsAAAAbIJgBwAAYBMEOwAAAJsg2AEAANgEwQ4AAMAmCHYAAAA2QbADAACwCWdBF9i7d69Wr16tQ4cO6cyZMxozZoyaNWtmzTfGaMmSJfriiy90/vx51a5dW4MGDVJkZKRV59y5c5o3b562b98uh8Oh5s2bq3///vL19XXPXgEAANyCCnzG7tKlS6pSpYoGDhyY5/xVq1bp448/1uDBgzVlyhT5+Pho8uTJSk9Pt+q88cYb+vXXXzV+/Hg9++yz2rdvn956660b3wsAAAAUPNg1adJEjzzyiMtZuhzGGK1Zs0bdunXTnXfeqejoaD355JM6c+aMvvnmG0nS0aNHtWvXLg0bNkw1atRQ7dq1NWDAAG3ZskWnT58u/B4BAADcotx6j92JEyeUkpKihg0bWmX+/v6KiYnR/v37JUn79+9XQECAqlevbtVp0KCBHA6HfvrpJ3c2BwAA4JZS4HvsriUlJUWSFBwc7FIeHBxszUtJSVFQUJDLfE9PTwUGBlp1rpSRkaGMjAxr2uFwyM/Pz/rdzgqzfznL2v0YAcWFPgW4F33K/dwa7IrKihUrtGzZMmu6atWqmjZtmsLCwkqwVcXj8odOblRERIQbWgIgB30KcC/6lPu4NdiFhIRIks6ePauyZcta5WfPnlWVKlWsOqmpqS7LZWVl6dy5c9byV4qPj1dcXJw1nZPsk5OTlZmZ6b4dKIUSExNveFmHw6GIiAglJSXJGOPGVgG3JvoU4F70qfxxOp35Ppnl1mAXHh6ukJAQ7d692wpyaWlp+umnn3TvvfdKkmrWrKnz58/r559/VrVq1SRJe/bskTFGMTExea7Xy8tLXl5eec6z+wvBHftnjLH9cQKKE30KcC/6lPsUONhdvHhRSUlJ1vSJEyd0+PBhBQYGKjQ0VA888ID+93//V5GRkQoPD9f777+vsmXL6s4775QkVapUSY0bN9Zbb72lwYMHKzMzU/PmzVPLli1Vrlw59+0ZAADALabAwe7gwYOaMGGCNb1w4UJJUtu2bTVixAh16dJFly5d0ltvvaW0tDTVrl1bY8eOlbe3t7XM008/rblz52rixInWAMUDBgxww+4AAADcuhzmJj73mZyc7PK0bFGI/U9ska7/ejb22XjDyzocDkVGRioxMZFT3IAb0KcA96JP5Y+Xl1e+77Hju2IBAABsgmAHAABgEwQ7AAAAmyDYAQAA2ATBDgAAwCYIdgAAADZBsAMAALAJgh0AAIBNEOwAAABsgmAHAABgEwQ7AAAAmyDYAQAA2ATBDgAAwCYIdgAAADZBsAMAALAJgh0AAIBNEOwAAABsgmAHAABgEwQ7AAAAmyDYAQAA2ATBDgAAwCYIdgAAADZBsAMAALAJgh0AAIBNEOwAAABsgmAHAABgEwQ7AAAAmyDYAQAA2ATBDgAAwCYIdgAAADZBsAMAALAJZ0k3AMCtJfY/sSW6/Y19Npbo9gGgKHHGDgAAwCYIdgAAADZBsAMAALAJgh0AAIBNEOwAAABsgmAHAABgEwQ7AAAAmyDYAQAA2ATBDgAAwCbc/s0TI0aMUHJycq7ye++9V4MGDdJLL72kvXv3uszr2LGjhgwZ4u6mAAAA3FLcHuymTp2q7Oxsa/rIkSOaNGmS7rrrLqusQ4cO6tWrlzXt7e3t7mYAAADcctwe7IKCglymV65cqQoVKqhu3bpWmY+Pj0JCQty9aQAAgFua24Pd5TIzM7Vx40Z17txZDofDKt+4caM2btyokJAQ3X777Xr44Yfl4+NTlE0BAACwvSINdlu3btX58+d19913W2WtW7dWaGioypUrp19++UWLFi3S8ePHNWbMmKuuJyMjQxkZGda0w+GQn5+f9budFWb/cpa1+zECCoI+BZQe9Cn3K9Jgl5CQoMaNG6tcuXJWWceOHa3fb7vtNpUtW1YTJ05UUlKSIiIi8lzPihUrtGzZMmu6atWqmjZtmsLCwoqu8aVEZGRkoddxteMK3IroU0DpQ59ynyILdsnJyfruu++ueSZOkmJiYiTpmsEuPj5ecXFx1nROsk9OTlZmZqabWlw6JSYm3vCyDodDERERSkpKkjHGja0Cbl70KaD0oE/lj9PpzPfJrCILdgkJCQoODlbTpk2vWe/w4cOSpLJly161jpeXl7y8vPKcZ/cXgjv2zxhj++ME5Bd9Cih96FPuUyTBLjs7W+vXr1fbtm3l6elplSclJWnTpk1q2rSpAgMDdeTIES1YsEB16tRRdHR0UTQFAADgllEkwW737t06efKk2rVr57oxp1O7d+/WmjVrdOnSJZUvX17NmzdXt27diqIZAAAAt5QiCXaNGjXSkiVLcpWHhoZqwoQJRbFJAACAWx7fFQsAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATTnevcMmSJVq2bJlLWVRUlGbNmiVJSk9P18KFC7VlyxZlZGSoUaNGGjRokEJCQtzdFAAAgFuK24OdJFWuXFnPP/+8Ne3h8X8nBhcsWKAdO3Zo9OjR8vf319y5czV9+nS9/PLLRdEUAACAW0aRXIr18PBQSEiI9RMUFCRJSktL07p169SvXz/Vr19f1apV0/Dhw/Xjjz9q//79RdEUAACAW0aRnLFLSkrS0KFD5eXlpZo1a6pPnz4KDQ3Vzz//rKysLDVo0MCqW7FiRYWGhmr//v2qWbNmUTQHAADgluD2YFejRg0NHz5cUVFROnPmjJYtW6YXXnhB06dPV0pKipxOpwICAlyWCQ4OVkpKylXXmZGRoYyMDGva4XDIz8/P+t3OCrN/Ocva/RgBBUGfAkoP+pT7uT3YNWnSxPo9OjraCnpfffWVvL29b2idK1ascHkgo2rVqpo2bZrCwsIK3d7SLjIystDriIiIcENLAHugTwGlD33KfYrkUuzlAgICFBUVpaSkJDVs2FCZmZk6f/68y1m7s2fPXvOp2Pj4eMXFxVnTOck+OTlZmZmZRdb20iAxMfGGl3U4HIqIiFBSUpKMMW5sFXDzok8BpQd9Kn+cTme+T2YVebC7ePGikpKSFBsbq2rVqsnT01O7d+9WixYtJEnHjx/XyZMnr3l/nZeXl7y8vPKcZ/cXgjv2zxhj++ME5Bd9Cih96FPu4/Zgt3DhQt1xxx0KDQ3VmTNntGTJEnl4eKh169by9/dX+/bttXDhQgUGBsrf31/z5s1TzZo1eXACAACgkNwe7E6fPq3XX39dv//+u4KCglS7dm1NnjzZGvKkX79+cjgcmj59ujIzM60BigEAAFA4bg92o0aNuuZ8b29vDRo0iDAHAADgZnxXLAAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE043b3CFStWaOvWrTp27Ji8vb1Vs2ZN9e3bV1FRUVadl156SXv37nVZrmPHjhoyZIi7mwMAAHDLcHuw27t3r+677z5Vr15dWVlZWrx4sSZNmqQZM2bI19fXqtehQwf16tXLmvb29nZ3UwAAAG4pbg9248aNc5keMWKEBg0apJ9//ll169a1yn18fBQSEuLuzdtO30mZhVzDkUIt/d54t79EAABAESnyT+20tDRJUmBgoEv5xo0btXHjRoWEhOj222/Xww8/LB8fnzzXkZGRoYyMDGva4XDIz8/P+h1Fh+MLuynMazpnWfoF4B70Kfcr0mCXnZ2t+fPnq1atWrrtttus8tatWys0NFTlypXTL7/8okWLFun48eMaM2ZMnutZsWKFli1bZk1XrVpV06ZNU1hYWFE2H5IiIyNLugmAW7njNR0REeGGlgDIQZ9ynyINdnPnztWvv/6qiRMnupR37NjR+v22225T2bJlNXHiRCUlJeX5x42Pj1dcXJw1nZPsk5OTlZlZ2EuVuJbExMSSbgLgVoV5TTscDkVERCgpKUnGGDe2Crg10afyx+l05vtkVpEFu7lz52rHjh2aMGGCypcvf826MTExknTVYOfl5SUvL688l+WFULQ4vrAbd7ymjTH0DcCN6FPu4/Zx7Iwxmjt3rrZu3aoXXnhB4eHh113m8OHDkqSyZcu6uzkAAAC3DLefsZs7d642bdqkZ555Rn5+fkpJSZEk+fv7y9vbW0lJSdq0aZOaNm2qwMBAHTlyRAsWLFCdOnUUHR3t7uYAAADcMtwe7D799FNJfwxCfLnhw4fr7rvvltPp1O7du7VmzRpdunRJ5cuXV/PmzdWtWzd3NwUAAOCW4vZgt2TJkmvODw0N1YQJE9y9WQAAgFse3xULAABgEwQ7AAAAmyDYAQAA2ATBDgAAwCb4hncAt5S+kwr7bTVHCrX0e+N52wVQdDhjBwAAYBMEOwAAAJsg2AEAANgEN3sAAHATi/1PbIluf2OfjSW6fbjijB0AAIBNcMYOAIpR1OKKJbr9472Plej2ARQtztgBAADYBGfsAADATYuz4K44YwcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACbINgBAADYBMEOAADAJgh2AAAANkGwAwAAsAmCHQAAgE0Q7AAAAGzCWdINQOkWtbhiiW7/eO9jJbp9AABuJpyxAwAAsAmCHQAAgE0Q7AAAAGyCYAcAAGATBDsAAACb4KlYAABww/pOyizkGo4Uaul11Qu5eZvhjB0AAIBNEOwAAABsgmAHAABgEwQ7AAAAmyDYAQAA2ESJPRW7du1affjhh0pJSVF0dLQGDBigmJiYkmoOAADATa9Eztht2bJFCxcuVPfu3TVt2jRFR0dr8uTJOnv2bEk0BwAAwBZKJNh99NFH6tChg9q1a6dKlSpp8ODB8vb2VkJCQkk0BwAAwBaK/VJsZmamfv75Z3Xt2tUq8/DwUIMGDbR///48l8nIyFBGRoY17XA45OfnJ6ez6JtfL7xekW/jWiIzvEp0+wpvUqKb9/Iq4f2H29Gn6FNwL/qU/ftUQfJOsQe71NRUZWdnKyQkxKU8JCREx48fz3OZFStWaNmyZdZ0q1atNHLkSJUtW7YomypJWv346iLfRum2o0S3HlaiW0dRoE/Rp+Be9Cn61OVuiqdi4+PjNX/+fOtn8ODBLmfwkLcLFy7ob3/7my5cuFDSTQFsgT4FuBd9yv2K/YxdUFCQPDw8lJKS4lKekpKS6yxeDi8vLy4f3ABjjA4dOiRjTEk3BbAF+hTgXvQp9yv2M3ZOp1PVqlXTnj17rLLs7Gzt2bNHNWvWLO7mAAAA2EaJjGMXFxenOXPmqFq1aoqJidGaNWt06dIl3X333SXRHAAAAFsokWDXsmVLpaamasmSJUpJSVGVKlU0duzYq16KxY3x8vJS9+7duYwNuAl9CnAv+pT7OQwXtgEAAGzhpngqFgAAANdHsAMAALAJgh0AAIBNEOwAAABsokSeisXVzZkzRxs2bMhV3qhRI40bN04jRoxQcnKyRo4cqVatWrnUGT16tI4eParhw4fnGjpmxYoVev/99/Xoo4/qoYcecpm3fv1661s98poGisOcOXN0/vx5PfPMMy79wNPTU6GhoWrbtq3i4+Pl6emZ73WOGjVKJ06c0JtvvpnrqfuXXnpJe/fulfTH+JplypRR1apV1a5dOzVv3tylbs+ePa3ffXx8VLZsWdWqVUudOnVStWrVrHnff/+9JkyYYE2XKVNGMTExevTRR3Xbbbe57Ou1+rkkHT58WB988IEOHDigCxcuKCQkRDExMRowYICCg4MlSVu3btWqVat09OhRGWMUGhqqhg0b6oknnsj3MQLc7fK+nOPrr7/W7Nmz9cgjj+jIkSNXff0/9dRT+stf/qJOnTqpW7duLvNnzJihU6dO6eWXX5aHB+elroZgVwo1btxYw4cPdym7/AuAy5cvr/Xr17sEu/379yslJUU+Pj55rjMhIUFdunRRQkJCrmAHlEY5/SAjI0M7d+7U3Llz5enpqfj4+Hwt/8MPPyg9PV0tWrTQ+vXr1bVr11x1OnTooF69eikrK0unTp3S1q1bNWvWLN19990aOnSoS93hw4ercePGSk9PV2Jioj7//HONHTtWf/rTn9S2bVuXurNmzZK/v79Onz6t9957T1OnTtXs2bNd+vG1+nlqaqpefvllNW3aVOPGjVNAQIBOnDihbdu26dKlS5Kk3bt3a+bMmerdu7dGjBghSTp69Ki+++67fB0foLh88cUXmjt3rgYPHqx27dppzpw5V339BwYGaujQoZoxY4buuOMO6x+ir776Sjt27NCrr75KqLsOjk4p5HQ6FRIS4vITGBhozY+NjdXevXt18uRJqywhIUGtW7fO82zG3r17lZ6erp49eyotLU0//vhjsewHUBg5/SAsLEz33nuvGjRooG3btuV7+XXr1ql169Zq06aNEhIS8qzj4+OjkJAQlS9fXjVr1lTfvn01ZMgQffHFF7kCkr+/v0JCQhQeHq5GjRrpL3/5i1q3bq158+bp3LlzLnWDg4MVEhKiatWq6YEHHtCpU6d07NixPPcvr37+ww8/KC0tTcOGDVPVqlUVHh6u+vXr64knnlB4eLgkafv27apdu7YeeughRUVFKSoqSs2aNdOgQYPyfYyAorZq1SrNmzdPo0aNUrt27azya73+77jjDrVu3Vpz5sxRZmamUlNTNXfuXPXp00dRUVEltSs3DYLdTSg4OFiNGjWyTmVfunRJW7ZsUfv27fOsv27dOrVq1UpOp1OtWrXSunXrirO5gFt4e3srMzMzX3UvXLigr776SrGxsWrYsKHS0tK0b9++fC3btm1bBQQEaOvWrdetGxcXpwsXLlz1LFlaWpq2bNkiyfWs+/WEhIQoKytLW7duvep3aIaEhOjXX3/VkSNH8r1eoDi99957Wr58uZ599lk1a9asQMs+8cQT+v3337V8+XK98847qly5sjp16lRELbUXLsWWQjt27NBjjz3mUhYfH+9yv0G7du20cOFCdevWTV9//bUiIiJUpUqVXOtKS0vT119/rUmTJkmS2rRpoxdeeEH9+/eXr69vke4H4A7GGO3evVvffvut7r///nwts3nzZkVGRqpy5cqSZP1DU6dOnesu6+HhoaioKCUnJ1+3bs7ZgyvrDhs2TJKsy6Z33HGHKlas6FLnWv28Zs2aio+P1xtvvKG3335bMTExql+/vtq0aWPdK3j//fdr3759GjNmjMLCwlSjRg01bNhQsbGxjOKPErdr1y5t27ZNL7zwgurXr59r/vU+5/z9/TV8+HBNmjRJPj4+eu211+RwOIql7Tc7gl0pVK9ePQ0ePNil7PJLsZLUtGlT/etf/9K+ffuUkJDgcor7cps3b1aFChWs0FelShWFhYVd8wwfUBrkvPFnZWXJGKNWrVqpR48e+Vo2ISFBsbGx1nRsbKxeeuklDRgwQH5+ftddvqBfyHPlB87EiRPl4+Oj/fv3a8WKFbn6s3T9ft67d2/FxcVpz549OnDggD777DOtWLFCEyZM0G233SZfX18999xzSkpK0vfff68DBw7o3//+tz7++GPrwxAoKdHR0dZXh8bExOQ6kZCfz7n69eurRo0a1ucW8odgVwr5+PgoIiLimnU8PT3Vpk0bLVmyRAcOHNCYMWPyrLdu3TodPXpUjzzyiFVmjFFCQgLBDqVazhu/0+lU2bJl8/007NGjR3XgwAH99NNPWrRokVWenZ2tzZs3q2PHjtdcPjs7W4mJiapevXq+tiXJuu8tR3h4uAICAhQVFaXU1FTNmjXL5WlZKX/9vEyZMrrrrrt01113qU+fPnrmmWe0evVqPfnkk1adiIgIRUREqEOHDurWrZtGjhypLVu2XPWfPaA4lC1bVqNHj9aECRM0efJkjR071uWfqvy8/qU/PusK8iQ8CHY3tXbt2unDDz9Uy5Ytc/2nI0lHjhzRzz//rBdffNFl/rlz5zRhwgQdO3Ys1+UhoLTI7xv/lXIuuQ4cONClfP369UpISLhusFu/fr3Onz+fa8iTvKxZs0Z+fn5q0KDBVevcd999WrFihbZu3Vrg+4wu53Q6VaFCBevybl7CwsLk7e19zTpAcQkLC9NLL72kCRMmaMqUKbnCHYoGwa4UyszMVEpKikuZh4eHgoKCXMoqVaqkuXPnXvWSy7p16xQTE6O6devmmle9enWtW7cu1z0OObKzs3X48GGXMqfTqUqVKuV/R4BilpmZqS+//FI9e/Z0GTdOktq3b6+PPvpIv/76q3Xv3aVLl5SSkuIy3Ml///tf3XvvvbnuC0pLS1NKSooyMjKUmJiozz77TN98842efPJJBQQEXLVNPj4+6tChg5YsWaI777zTumx7rX6+fft2bd68Wa1atVJkZKQkadu2bdq5c6c1RMSSJUuUnp6uJk2aKCwsTOfPn9fHH3+srKwsNWzYsFDHEXCX0NBQK9zlnLmT8v85h4Ij2JVCu3bt0pAhQ1zKoqKiNGvWrFx1y5Qpk+c6MjMztXHjRnXp0iXP+c2bN9dHH32k3r175zn/4sWLLoNLSlKFChU0e/bsfOwBUDK2bdum33//Pc8zY5UqVVLFihW1bt069evXT9If42t98cUXLgMU//nPf85z+TfffFOS5OXlpXLlyql27dqaMmWKywDFV3P//ffrv//9r7766iu1bNlS0rX7eaVKleTj46OFCxfq1KlT8vLyUkREhIYNG6Y2bdpIkurWratPPvlE//jHP3T27FkFBASoatWqGj9+PENCoFQpX768S7grW7ZsgT7nUDAOU9C7hAEAAFAqMY4dAACATXApFsBNZcqUKVcdbPjK8R4B4FbDpVgAN5XTp08rPT09z3mBgYF5PiEOALcKgh0AAIBNcI8dAACATRDsAAAAbIJgBwAAYBMEOwAAAJsg2AEAANgEwQ4AAMAmCHYAAAA2QbADAACwif8fXanY6mkYwZ4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot number TP, FP, FN for each tag\n",
    "# use ggplot style\n",
    "plt.style.use('ggplot')\n",
    "tags = [\"EMAIL\", \"IP_ADDRESS\", \"KEY\"]\n",
    "values = [metrics_dict_2[tag] for tag in tags]\n",
    "tp = [v[\"TP\"] for v in values]\n",
    "fp = [v[\"FP\"] for v in values]\n",
    "fn = [v[\"FN\"] for v in values]\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "x = np.arange(len(tags))  # the label locations\n",
    "width = 0.15  # the width of the bars\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "# use mat\n",
    "# use green, dark orange and royal blue and bars shouldn't overlap\n",
    "rects1 = ax.bar(x - width, tp, width, label='TP', color=\"#228B22\")\n",
    "rects2 = ax.bar(x, fp, width, label='FP', color=\"#4169E1\")\n",
    "rects3 = ax.bar(x + width, fn, width, label='FN', color=\"#FF8C00\")\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "# use Mathematica text in title and increase size\n",
    "ax.set_title('Number of TP, FP, FN samples for each tag', fontname=\"STIXGeneral\", fontsize=16)\n",
    "\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(tags)\n",
    "ax.legend()\n",
    "\n",
    "fig.tight_layout()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot precision and recall of each tag from metrics_2\n",
    "tags = [\"EMAIL\", \"IP_ADDRESS\", \"KEY\"]\n",
    "values = [metrics_2[tag] for tag in tags]\n",
    "precision = [v[\"precision\"] for v in values]\n",
    "recall = [v[\"recall\"] for v in values]\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "x = np.arange(len(tags))  # the label locations\n",
    "width = 0.15  # the width of the bars\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8,5), dpi=140)\n",
    "c0 = np.array([57, 62, 107])/255*1.5\n",
    "c1 = np.array([255, 117, 180])/255\n",
    "# use c0 and c1 as colors\n",
    "rects1 = ax.bar(x - width/2, precision, width, label='Precision', color=c0)\n",
    "rects2 = ax.bar(x + width/2, recall, width, label='Recall', color=c1)\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "\n",
    "ax.set_title('Precision and Recall for each entity', fontname=\"STIXGeneral\", fontsize=16)\n",
    "ax.set_ylim(0, 1)\n",
    "ax.set_yticks([0.2, 0.4, 0.6, 0.8, 1])\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(tags)\n",
    "ax.set_axisbelow(True)\n",
    "ax.grid(True, which=\"major\", axis=\"y\")\n",
    "ax.grid(True, which=\"minor\", axis=\"y\", linestyle='dashed')\n",
    "ax.legend()\n",
    "\n",
    "fig.tight_layout()\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.4 ('venv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "fd8fde6f83dada9276d12fdb71d773558994168ed1b3bea457b8db38c02aa2e1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
